package Image;


import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;

/*
 * This class shows how you can crawl images on the web and store them in a
 * folder. This is just for demonstration purposes and doesn't scale for large
 * number of images. For crawling millions of images you would need to store
 * downloaded images in a hierarchy of folders
 */
public class ImageCrawler extends WebCrawler {

    private static final Pattern filters = Pattern.compile(".*(\\.(css|js|mid|mp2|mp3|mp4|wav|avi|mov|mpeg|ram|m4v|pdf"
            + "|rm|smil|wmv|swf|wma|zip|rar|gz))$");

    private static final Pattern imgPatterns = Pattern.compile(".*(\\.(bmp|gif|jpe?g|png|tiff?))$");
    static FileOutputStream fop;

    private static String crawlDomains;
    private Set<WebURL> webURLSet;



    @Override
    public void onStart() {
        crawlDomains = (String) getMyController().getCustomData();
        webURLSet = new HashSet<WebURL>();
    }

    @Override
    public boolean shouldVisit(WebURL url) {
        String href = url.getURL().toLowerCase();
        return !filters.matcher(href).matches() && href.startsWith(crawlDomains);
    }

    @Override
    public void visit(Page page) {

        System.out.println("URL: " + page.getWebURL());
        this.webURLSet.add(page.getWebURL());
        if (page.getParseData() instanceof HtmlParseData) {
            HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
            String text = htmlParseData.getText();
            String html = htmlParseData.getHtml();
            List<WebURL> links = htmlParseData.getOutgoingUrls();
            System.out.println("==================================");
            System.out.println("Text length: " + text.length());
            System.out.println("Html length: " + html.length());
            System.out.println("Number of outgoing links: " + links.size());
            System.out.println("==================================");
            /*try {
                fop.write(("==================================\n").getBytes());
                fop.write(("URL: " + url + "\n").getBytes());
                fop.write(("Text length: " + text.length() + "\n").getBytes());
                fop.write(("Html length: " + html.length() + "\n").getBytes());
                fop.write(("Number of outgoing links: " + links.size() + "\n").getBytes());
                fop.write(("==================================" + "\n").getBytes());
            } catch (IOException e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
            System.out.println("Text length: " + text.length());
            System.out.println("Html length: " + html.length());
            //System.out.println(html);
            System.out.println("Number of outgoing links: " + links.size());*/
        }

    }

    @Override
    public void onBeforeExit() {
        super.onBeforeExit();
        for(WebURL web:webURLSet){
            System.out.println("Visitou " + web.getURL() + " no " + getMyId());
        }
    }

	/*String url = page.getWebURL().getURL();

		System.out.println(page.getWebURL());

		System.out.println(page.getParseData());
		// We are only interested in processing images
		if (!(page.getParseData() instanceof BinaryParseData)) {
			return;
		}

		if (!imgPatterns.matcher(url).matches()) {
			return;
		}

		// Not interested in very small images
		if (page.getContentData().length < 10 * 1024) {
			return;
		}

		// get a unique name for storing this image
		String extension = url.substring(url.lastIndexOf("."));
		String hashedName = Cryptography.MD5(url) + extension;
		// store image
		IO.writeBytesToFile(page.getContentData(), storageFolder.getAbsolutePath() + "/" + hashedName);

		System.out.println("Stored: " + url);*/
}
